import os
current_directory = os.getcwd()
print(current_directory)
/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification
import numpy
numpy.version.version
'1.22.0'
# importing libraries
from tensorflow import keras
import tensorflow as tf
from tensorflow.keras.preprocessing.text import Tokenizer
from keras.utils import pad_sequences
from tensorflow.keras.preprocessing import text, sequence
from tensorflow.keras import initializers, regularizers, constraints, optimizers, layers, callbacks
from tensorflow.python.keras.models import Model, Input
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import GRU, Dense, Input, LSTM, Embedding, Dropout, SpatialDropout1D, Activation, SimpleRNN
from tensorflow.keras.layers import Conv1D, Bidirectional, GlobalMaxPool1D, MaxPooling1D, BatchNormalization, Add, Flatten
from tensorflow.keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate
from tensorflow.keras.optimizers import Adam
#from tensorflow.keras.optimizers import SGD
# For custom metrics
import keras.backend as K
from keras.utils.vis_utils import plot_model
from keras.callbacks import EarlyStopping, ModelCheckpoint
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn')
import seaborn as sns
from IPython.display import Image
from tqdm import tqdm
from nltk.corpus import stopwords
from nltk.tokenize import RegexpTokenizer
import os, re, csv, math, codecs
from nltk.tokenize import word_tokenize
import string
import gensim
sns.set_style("whitegrid")
np.random.seed(0)
# Install dependencies
!apt install graphviz
!pip install pydot pydot-ng
!echo "Double check with Python 3"
!python -c "import pydot"
2023-04-02 04:07:49.749013: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations. To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.
The operation couldn’t be completed. Unable to locate a Java Runtime that supports apt. Please visit http://www.java.com for information on installing Java. Requirement already satisfied: pydot in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (1.4.2) Requirement already satisfied: pydot-ng in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (2.0.0) Requirement already satisfied: pyparsing>=2.1.4 in /Users/macbookpro/opt/anaconda3/lib/python3.9/site-packages (from pydot) (3.0.4) Double check with Python 3
df = pd.read_csv('/Users/macbookpro/hdd/MSc/Dissertation/multilabeltextclassification/githubissuedata.csv')
df.info()
<class 'pandas.core.frame.DataFrame'> RangeIndex: 226163 entries, 0 to 226162 Data columns (total 17 columns): # Column Non-Null Count Dtype --- ------ -------------- ----- 0 title 225152 non-null object 1 body 225866 non-null object 2 task 226163 non-null int64 3 bug 226163 non-null int64 4 documentation 226163 non-null int64 5 duplicate 226163 non-null int64 6 enhancement 226163 non-null int64 7 good_first_issue 226163 non-null int64 8 help_wanted 226163 non-null int64 9 invalid 226163 non-null int64 10 question 226163 non-null int64 11 wontfix 226163 non-null int64 12 gitalk 226163 non-null int64 13 priority_medium 226163 non-null int64 14 priority_high 226163 non-null int64 15 feature_request 226163 non-null int64 16 feature 226163 non-null int64 dtypes: int64(15), object(2) memory usage: 29.3+ MB
df.head(10)
| title | body | task | bug | documentation | duplicate | enhancement | good_first_issue | help_wanted | invalid | question | wontfix | gitalk | priority_medium | priority_high | feature_request | feature | |
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
| 0 | My Account Paid laptop 1440 resolution Updat... | Case:Distance between Registered email address... | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 1 | How to fix sleepimpl warning when ECS credenti... | Prerequisites X Ive searched for previous sim... | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 2 | Slider doesnt work on touch devices | DescriptionSlider should work dragging in tou... | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 3 | add new labels | DescriptionAdd ui and logic to permanently ad... | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 4 | No lib sub folder in Boost folder | Hi I am following thishttps://www.mlpack.org/d... | 0 | 0 | 0 | 0 | 0 | 0 | 1 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |
| 5 | Add license notice to CLI | The CLI is missing the license notice. Theres ... | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1 |
| 6 | Should show Powershell or AzureCLI code necess... | There is example output from Powershell and Az... | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 7 | tidboperator could not work with kubernetes 1.23 | Bug ReportWhat version of Kubernetes are you ... | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 8 | Match Live | x Implement game logic x Calculate results ba... | 0 | 0 | 0 | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
| 9 | AngularBug Make current location widget more g... | If youve never submitted an issue to the SORMA... | 0 | 1 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 0 |
# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')
The dataset has 1308 missing values in test data. The dataset has 46411 duplicates in train data.
# remove missing values in the dataset
df.dropna(inplace= True)
# remove all duplicate records in the dataset
df.drop_duplicates(inplace= True)
# check missing values in the dataset
print('The dataset has', df.isna().sum().sum(), 'missing values in test data.')
# check any duplicate records in the dataset
print('The dataset has', df.duplicated().sum(), 'duplicates in train data.')
The dataset has 0 missing values in test data. The dataset has 0 duplicates in train data.
# spliting dataset to train and test
from sklearn.model_selection import train_test_split
train_df, test_df = train_test_split(df, test_size=0.2, random_state=25)
print("Train data shape", train_df.shape)
print("Test data shape", test_df.shape)
Train data shape (143145, 17) Test data shape (35787, 17)
fig, ax = plt.subplots(figsize=(10, 6))
fig.suptitle('Correlation Matrix')
sns.heatmap(train_df.corr(), annot=True, cmap="Greens", linewidths=.5, ax=ax);
The correlation figure below shows that Toxic" comments are clearly correlated with both "obscene" and "insult" comments. Interestingly, "toxic" and "severe_toxic" are only weakly correlated. While we can also observe that, "Obscene" comments and "insult" comments are also highly correlated, which makes perfect sense.
Deep Neural Networks input layers make use of input variables to feed the network for training the model. But in this task (experiment), we're dealing with words text. How do we represent these words in order to feed our model?
In our experiment, we used densed representation of those text (comments) and their semanticity together. The advantage of using this approach is the best way for fitting neural networks onto a text data (as in our case), as well as less memory usage compared to other sparse representation approaches.
Two ways to feed embeddings to neural networks:
#Convert text to vectors using keras preprocessing library tools
X_train = train_df["body"].values
X_test = test_df["body"].values
y_train = train_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
y_test = test_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
For the first embedding, we used keras preprocessing (Text Preprocessing) libraries. This class allows to vectorize a text corpus, by turning each text into either a sequence of integers (each integer being the index of a token in a dictionary) or into a vector where the coefficient for each token could be binary, based on word count, based on tf-idf
num_words = 20000 #Max. words to use per issue label
max_features = 200000 #Max. number of unique words in embeddinbg vector
max_len = 500 #Max. number of words per issue to be use
embedding_dims = 128 #embedding vector output dimension
num_epochs = 15 # (before 5)number of epochs (number of times that the model is exposed to the training dataset)
val_split = 0.1
batch_size2 = 256 #(before 32)The **batch size** is the number of training examples in one forward/backward pass.
# In general, larger batch sizes result in faster progress in training, but don't always converge as quickly.
#Smaller batch sizes train slower, but can converge faster. And the higher the batch size, the more memory space you’ll need.
#Issue body Tokenization
tokenizer = tokenizer = Tokenizer(num_words)
tokenizer.fit_on_texts(list(X_train))
#Convert tokenized issue body text to sequnces
X_train = tokenizer.texts_to_sequences(X_train)
X_test = tokenizer.texts_to_sequences(X_test)
# padding the sequences
X_train = pad_sequences(X_train, max_len)
X_test = pad_sequences(X_test, max_len)
print('X_train shape:', X_train.shape)
print('X_test shape: ', X_test.shape)
X_train shape: (143145, 500) X_test shape: (35787, 500)
##We use cross validation to split arrays or matrices of train data into random train and validation subsets
X_tra, X_val, y_tra, y_val = train_test_split(X_train, y_train, train_size =0.9, random_state=233)
#We used early callback functionality that allows you to specify the performance measure to monitor, the trigger, and once triggered. It will stop the training process.
early = EarlyStopping(monitor="val_loss", mode="min", patience=4)
#Writing functions for Precision, Recall, F1-Measure, AUC, mean etc evaluaiton metrics to evaluate the model
#Import necessary libraries
4# demonstration of calculating metrics for a neural network model using sklearn
from sklearn.datasets import make_circles
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_score
from sklearn.metrics import recall_score
from sklearn.metrics import f1_score
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import roc_auc_score
from sklearn.metrics import confusion_matrix
def precision(y_true, y_pred):
#Calculating precision, a metric for multi-label classification of how many selected items are relevant.
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))
precision = true_positives / (predicted_positives + K.epsilon())
return precision
def recall(y_true, y_pred):
#Calculating recall, a metric for multi-label classification of how many relevant items are selected.
true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))
possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))
recall = true_positives / (possible_positives + K.epsilon())
return recall
#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
return K.mean(y_pred)
def fbeta_score(y_true, y_pred, beta=1):
'''Calculates the F score, the weighted harmonic mean of precision and recall.
This is useful for multi-label classification, where input samples can be
classified as sets of labels. By only using accuracy (precision) a model
would achieve a perfect score by simply assigning every class to every
input. In order to avoid this, a metric should penalize incorrect class
assignments as well (recall). The F-beta score (ranged from 0.0 to 1.0)
computes this, as a weighted mean of the proportion of correct class
assignments vs. the proportion of incorrect class assignments.
With beta = 1, this is equivalent to a F-measure. With beta < 1, assigning
correct classes becomes more important, and with beta > 1 the metric is
instead weighted towards penalizing incorrect class assignments.
'''
if beta < 0:
raise ValueError('The lowest choosable beta is zero (only precision).')
# If there are no true positives, fix the F score at 0 like sklearn.
if K.sum(K.round(K.clip(y_true, 0, 1))) == 0:
return 0.0
p = precision(y_true, y_pred)
r = recall(y_true, y_pred)
bb = beta ** 2
fbeta_score = (1 + bb) * (p * r) / (bb * p + r + K.epsilon())
return fbeta_score
def auroc(y_true, y_pred):
auc = tf.keras.metrics.AUC(y_true, y_pred)[1]
#auc = tf.metrics.auc(y_true, y_pred)[1]
K.get_session().run(tf.local_variables_initializer())
return auc
def fmeasure(y_true, y_pred):
#Calculates the f-measure, the harmonic mean of precision and recall.
return fbeta_score(y_true, y_pred, beta=1)
fscore = f1score = fmeasure
#load embeddings
print('loading word embeddings...')
fastText_embeddings_index = {}
f = codecs.open('wiki.simple.vec', encoding='utf-8')
for line in tqdm(f):
values = line.rstrip().rsplit(' ')
word = values[0]
coefs = np.asarray(values[1:], dtype='float32')
fastText_embeddings_index[word] = coefs
f.close()
print('found %s word vectors' % len(fastText_embeddings_index))
loading word embeddings...
111052it [00:11, 9927.93it/s]
found 111052 word vectors
import nltk
nltk.download('punkt')
nltk.download('stopwords')
issueBodies_lines = list()
lines = train_df['body'].values.tolist()
for line in lines:
tokens = word_tokenize(line)
#convert to lower case
tokens = [w.lower() for w in tokens]
#remove punctuation from each word
table = str.maketrans('', '', string.punctuation)
stripped = [w.translate(table) for w in tokens]
#remove remaining tpkens gthat are not alphabetic
words = [word for word in stripped if word.isalpha()]
#filter out stop words
stop_words = set(stopwords.words('english'))
words = [w for w in words if not w in stop_words]
issueBodies_lines.append(words)
len(issueBodies_lines)
[nltk_data] Downloading package punkt to [nltk_data] /Users/macbookpro/nltk_data... [nltk_data] Package punkt is already up-to-date! [nltk_data] Downloading package stopwords to [nltk_data] /Users/macbookpro/nltk_data... [nltk_data] Package stopwords is already up-to-date!
143145
#vectorize the text samples into a 2D integer tensor
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(issueBodies_lines)
sequences = tokenizer_obj.texts_to_sequences(issueBodies_lines)
#pad sequences
word_index = tokenizer_obj.word_index
print('Found %s uniquue tokens.' % len(word_index))
issueBodies_pad = pad_sequences(sequences, maxlen=max_len)
issue_tag = train_df[["task","bug","documentation","duplicate","enhancement","good_first_issue","help_wanted","invalid","question","wontfix","gitalk","priority_medium","priority_high","feature_request","feature"]].values
print('Shape of issue bodies tensor', issueBodies_pad.shape)
print('Shape of issue bodies tensor', issue_tag.shape)
Found 792007 uniquue tokens. Shape of issue bodies tensor (143145, 500) Shape of issue bodies tensor (143145, 15)
#embedding matrix
print('preparing embedding matrix...')
max_nb_words = 100000
fastText_embed_dim = 300
words_not_found = []
nb_words = min(max_nb_words, len(word_index))
fastText_embedding_matrix = np.zeros((nb_words, fastText_embed_dim))
for word, i in word_index.items():
if i >= nb_words:
continue
fastText_embedding_vector = fastText_embeddings_index.get(word)
if (fastText_embedding_vector is not None) and len(fastText_embedding_vector) > 0:
# words not found in embedding index will be all-zeros.
fastText_embedding_matrix[i] = fastText_embedding_vector
else:
words_not_found.append(word)
print('number of null word embeddings: %d' % np.sum(np.sum(fastText_embedding_matrix, axis=1) == 0))
preparing embedding matrix... number of null word embeddings: 80365
print(nb_words)
100000
#1 Convolutional Neural Network (CNN) with fastText
CNN_FastText_model = Sequential([
Embedding(input_dim=fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
SpatialDropout1D(0.5),
# ... 100 filters with a kernel size of 4 so that each convolution will consider a window of 4 word embeddings
Conv1D(filters=100, kernel_size=4, padding='same', activation='relu'),
#**batch normalization layer** normalizes the activations of the previous layer at each batch,
#i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1.
#It will be added after the activation function between a convolutional and a max-pooling layer.
BatchNormalization(),
GlobalMaxPool1D(),
Dropout(0.5),
Dense(50, activation = 'relu'),
Dense(15, activation = 'sigmoid')
])
#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
return K.mean(y_pred)
CNN_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])
CNN_FastText_model.summary()
Model: "sequential"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding (Embedding) (None, 500, 300) 30000000
spatial_dropout1d (SpatialD (None, 500, 300) 0
ropout1D)
conv1d (Conv1D) (None, 500, 100) 120100
batch_normalization (BatchN (None, 500, 100) 400
ormalization)
global_max_pooling1d (Globa (None, 100) 0
lMaxPooling1D)
dropout (Dropout) (None, 100) 0
dense (Dense) (None, 50) 5050
dense_1 (Dense) (None, 15) 765
=================================================================
Total params: 30,126,315
Trainable params: 126,115
Non-trainable params: 30,000,200
_________________________________________________________________
CNN_FastText_model_fit = CNN_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15 504/504 [==============================] - 327s 646ms/step - loss: 0.2166 - accuracy: 0.5015 - mean_pred: 0.0862 - fmeasure: 0.3444 - precision: 0.6240 - recall: 0.2450 - val_loss: 0.1876 - val_accuracy: 0.5667 - val_mean_pred: 0.0881 - val_fmeasure: 0.3272 - val_precision: 0.8626 - val_recall: 0.2024 Epoch 2/15 504/504 [==============================] - 319s 633ms/step - loss: 0.1814 - accuracy: 0.5644 - mean_pred: 0.0820 - fmeasure: 0.4403 - precision: 0.7137 - recall: 0.3193 - val_loss: 0.1746 - val_accuracy: 0.5969 - val_mean_pred: 0.0875 - val_fmeasure: 0.4871 - val_precision: 0.7437 - val_recall: 0.3626 Epoch 3/15 504/504 [==============================] - 319s 632ms/step - loss: 0.1771 - accuracy: 0.5831 - mean_pred: 0.0815 - fmeasure: 0.4741 - precision: 0.7072 - recall: 0.3576 - val_loss: 0.1711 - val_accuracy: 0.6039 - val_mean_pred: 0.0825 - val_fmeasure: 0.5201 - val_precision: 0.7187 - val_recall: 0.4080 Epoch 4/15 504/504 [==============================] - 347s 688ms/step - loss: 0.1746 - accuracy: 0.5941 - mean_pred: 0.0812 - fmeasure: 0.4936 - precision: 0.7098 - recall: 0.3796 - val_loss: 0.1691 - val_accuracy: 0.6151 - val_mean_pred: 0.0865 - val_fmeasure: 0.5398 - val_precision: 0.7121 - val_recall: 0.4351 Epoch 5/15 504/504 [==============================] - 320s 635ms/step - loss: 0.1723 - accuracy: 0.6004 - mean_pred: 0.0812 - fmeasure: 0.5047 - precision: 0.7106 - recall: 0.3926 - val_loss: 0.1659 - val_accuracy: 0.6196 - val_mean_pred: 0.0834 - val_fmeasure: 0.5554 - val_precision: 0.7235 - val_recall: 0.4512 Epoch 6/15 504/504 [==============================] - 354s 702ms/step - loss: 0.1701 - accuracy: 0.6061 - mean_pred: 0.0810 - fmeasure: 0.5135 - precision: 0.7175 - recall: 0.4010 - val_loss: 0.1643 - val_accuracy: 0.6155 - val_mean_pred: 0.0833 - val_fmeasure: 0.5494 - val_precision: 0.7305 - val_recall: 0.4407 Epoch 7/15 504/504 [==============================] - 321s 637ms/step - loss: 0.1686 - accuracy: 0.6094 - mean_pred: 0.0810 - fmeasure: 0.5183 - precision: 0.7217 - recall: 0.4056 - val_loss: 0.1618 - val_accuracy: 0.6362 - val_mean_pred: 0.0818 - val_fmeasure: 0.5379 - val_precision: 0.7578 - val_recall: 0.4173 Epoch 8/15 504/504 [==============================] - 319s 634ms/step - loss: 0.1675 - accuracy: 0.6112 - mean_pred: 0.0808 - fmeasure: 0.5214 - precision: 0.7249 - recall: 0.4086 - val_loss: 0.1625 - val_accuracy: 0.6247 - val_mean_pred: 0.0843 - val_fmeasure: 0.5335 - val_precision: 0.7632 - val_recall: 0.4105 Epoch 9/15 504/504 [==============================] - 318s 631ms/step - loss: 0.1665 - accuracy: 0.6148 - mean_pred: 0.0808 - fmeasure: 0.5275 - precision: 0.7247 - recall: 0.4158 - val_loss: 0.1611 - val_accuracy: 0.6370 - val_mean_pred: 0.0830 - val_fmeasure: 0.5549 - val_precision: 0.7451 - val_recall: 0.4425 Epoch 10/15 504/504 [==============================] - 320s 635ms/step - loss: 0.1656 - accuracy: 0.6176 - mean_pred: 0.0809 - fmeasure: 0.5312 - precision: 0.7266 - recall: 0.4197 - val_loss: 0.1612 - val_accuracy: 0.6301 - val_mean_pred: 0.0804 - val_fmeasure: 0.5404 - val_precision: 0.7661 - val_recall: 0.4179 Epoch 11/15 504/504 [==============================] - 319s 633ms/step - loss: 0.1649 - accuracy: 0.6202 - mean_pred: 0.0808 - fmeasure: 0.5347 - precision: 0.7290 - recall: 0.4231 - val_loss: 0.1598 - val_accuracy: 0.6357 - val_mean_pred: 0.0835 - val_fmeasure: 0.5726 - val_precision: 0.7246 - val_recall: 0.4737 Epoch 12/15 504/504 [==============================] - 312s 619ms/step - loss: 0.1644 - accuracy: 0.6213 - mean_pred: 0.0808 - fmeasure: 0.5381 - precision: 0.7281 - recall: 0.4278 - val_loss: 0.1601 - val_accuracy: 0.6328 - val_mean_pred: 0.0854 - val_fmeasure: 0.5667 - val_precision: 0.7469 - val_recall: 0.4570 Epoch 13/15 504/504 [==============================] - 323s 640ms/step - loss: 0.1637 - accuracy: 0.6237 - mean_pred: 0.0808 - fmeasure: 0.5435 - precision: 0.7268 - recall: 0.4348 - val_loss: 0.1584 - val_accuracy: 0.6397 - val_mean_pred: 0.0821 - val_fmeasure: 0.5758 - val_precision: 0.7348 - val_recall: 0.4738 Epoch 14/15 504/504 [==============================] - 314s 622ms/step - loss: 0.1632 - accuracy: 0.6254 - mean_pred: 0.0808 - fmeasure: 0.5465 - precision: 0.7286 - recall: 0.4379 - val_loss: 0.1595 - val_accuracy: 0.6320 - val_mean_pred: 0.0845 - val_fmeasure: 0.5855 - val_precision: 0.7038 - val_recall: 0.5016 Epoch 15/15 504/504 [==============================] - 315s 626ms/step - loss: 0.1629 - accuracy: 0.6271 - mean_pred: 0.0808 - fmeasure: 0.5482 - precision: 0.7269 - recall: 0.4411 - val_loss: 0.1576 - val_accuracy: 0.6433 - val_mean_pred: 0.0822 - val_fmeasure: 0.5795 - val_precision: 0.7392 - val_recall: 0.4769
# Plot training & validation accuracy values
plt.plot(CNN_FastText_model_fit.history['accuracy'])
plt.plot(CNN_FastText_model_fit.history['val_accuracy'])
plt.title('CNN-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(CNN_FastText_model_fit.history['loss'])
plt.plot(CNN_FastText_model_fit.history['val_loss'])
plt.title('CNN-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
#2 Recurrent Neural Network (RNN) with fastText
RNN_FastText_model = Sequential([
Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
SpatialDropout1D(0.5),
#Fully-connected RNN where the output is to be fed back to input.
SimpleRNN(25, return_sequences=True),
#**batch normalization layer** normalizes the activations of the previous layer at each batch,
#i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1.
BatchNormalization(),
Dropout(0.5),
GlobalMaxPool1D(),
Dense(50, activation = 'relu'),
Dense(15, activation = 'sigmoid')
])
#Customized the evaluation to analyse the model in terms of accuracy and mean value accuracy
def mean_pred(y_true, y_pred):
return K.mean(y_pred)
RNN_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])
RNN_FastText_model.summary()
Model: "sequential_1"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_1 (Embedding) (None, 500, 300) 30000000
spatial_dropout1d_1 (Spatia (None, 500, 300) 0
lDropout1D)
simple_rnn (SimpleRNN) (None, 500, 25) 8150
batch_normalization_1 (Batc (None, 500, 25) 100
hNormalization)
dropout_1 (Dropout) (None, 500, 25) 0
global_max_pooling1d_1 (Glo (None, 25) 0
balMaxPooling1D)
dense_2 (Dense) (None, 50) 1300
dense_3 (Dense) (None, 15) 765
=================================================================
Total params: 30,010,315
Trainable params: 10,265
Non-trainable params: 30,000,050
_________________________________________________________________
RNN_FastText_model_fit = RNN_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15 504/504 [==============================] - 199s 391ms/step - loss: 0.1996 - accuracy: 0.5055 - mean_pred: 0.0830 - fmeasure: 0.3450 - precision: 0.6646 - recall: 0.2442 - val_loss: 0.2351 - val_accuracy: 0.5438 - val_mean_pred: 0.1578 - val_fmeasure: 0.3763 - val_precision: 0.7679 - val_recall: 0.2496 Epoch 2/15 504/504 [==============================] - 198s 394ms/step - loss: 0.1868 - accuracy: 0.5263 - mean_pred: 0.0812 - fmeasure: 0.3700 - precision: 0.7290 - recall: 0.2490 - val_loss: 0.2108 - val_accuracy: 0.5425 - val_mean_pred: 0.1341 - val_fmeasure: 0.4122 - val_precision: 0.7260 - val_recall: 0.2881 Epoch 3/15 504/504 [==============================] - 197s 391ms/step - loss: 0.1840 - accuracy: 0.5410 - mean_pred: 0.0810 - fmeasure: 0.3944 - precision: 0.7293 - recall: 0.2720 - val_loss: 0.2088 - val_accuracy: 0.5751 - val_mean_pred: 0.1370 - val_fmeasure: 0.4300 - val_precision: 0.7677 - val_recall: 0.2990 Epoch 4/15 504/504 [==============================] - 197s 391ms/step - loss: 0.1802 - accuracy: 0.5601 - mean_pred: 0.0809 - fmeasure: 0.4341 - precision: 0.7165 - recall: 0.3128 - val_loss: 0.2038 - val_accuracy: 0.5809 - val_mean_pred: 0.1273 - val_fmeasure: 0.3956 - val_precision: 0.8226 - val_recall: 0.2609 Epoch 5/15 504/504 [==============================] - 198s 393ms/step - loss: 0.1792 - accuracy: 0.5653 - mean_pred: 0.0809 - fmeasure: 0.4425 - precision: 0.7208 - recall: 0.3203 - val_loss: 0.2021 - val_accuracy: 0.5868 - val_mean_pred: 0.1281 - val_fmeasure: 0.4608 - val_precision: 0.7668 - val_recall: 0.3298 Epoch 6/15 504/504 [==============================] - 199s 395ms/step - loss: 0.1784 - accuracy: 0.5683 - mean_pred: 0.0809 - fmeasure: 0.4498 - precision: 0.7198 - recall: 0.3283 - val_loss: 0.1910 - val_accuracy: 0.5785 - val_mean_pred: 0.1072 - val_fmeasure: 0.4530 - val_precision: 0.6723 - val_recall: 0.3419 Epoch 7/15 504/504 [==============================] - 196s 388ms/step - loss: 0.1955 - accuracy: 0.5013 - mean_pred: 0.0810 - fmeasure: 0.1991 - precision: 0.7152 - recall: 0.1574 - val_loss: 0.2093 - val_accuracy: 0.4957 - val_mean_pred: 0.1114 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068 Epoch 8/15 504/504 [==============================] - 198s 394ms/step - loss: 0.1970 - accuracy: 0.4926 - mean_pred: 0.0808 - fmeasure: 0.0900 - precision: 0.7832 - recall: 0.0664 - val_loss: 0.2134 - val_accuracy: 0.4957 - val_mean_pred: 0.1165 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068 Epoch 9/15 504/504 [==============================] - 199s 395ms/step - loss: 0.1970 - accuracy: 0.4926 - mean_pred: 0.0809 - fmeasure: 0.0921 - precision: 0.7828 - recall: 0.0685 - val_loss: 0.2068 - val_accuracy: 0.4957 - val_mean_pred: 0.1029 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068 Epoch 10/15 504/504 [==============================] - 197s 391ms/step - loss: 0.1969 - accuracy: 0.4926 - mean_pred: 0.0809 - fmeasure: 0.0835 - precision: 0.7927 - recall: 0.0603 - val_loss: 0.2064 - val_accuracy: 0.4957 - val_mean_pred: 0.1005 - val_fmeasure: 0.4471 - val_precision: 0.4964 - val_recall: 0.4068
# Plot training & validation accuracy values
plt.plot(RNN_FastText_model_fit.history['accuracy'])
plt.plot(RNN_FastText_model_fit.history['val_accuracy'])
plt.title('RNN-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(RNN_FastText_model_fit.history['loss'])
plt.plot(RNN_FastText_model_fit.history['val_loss'])
plt.title('RNN-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
#3 LSTM with fastText
LSTM_FastText_model = Sequential([
Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
SpatialDropout1D(0.5),
#Bidirectional layer will enable our model to predict a missing word in a sequence,
#So, using this feature will enable the model to look at the context on both the left and the right.
LSTM(25, return_sequences=True),
#**batch normalization layer** normalizes the activations of the previous layer at each batch,
#i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1.
BatchNormalization(),
Dropout(0.5),
GlobalMaxPool1D(),
Dense(50, activation = 'relu'),
Dense(15, activation = 'sigmoid')
])
LSTM_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])
2023-04-02 14:55:17.699192: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:17.701077: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:17.703066: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
LSTM_FastText_model.summary()
Model: "sequential_3"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_3 (Embedding) (None, 500, 300) 30000000
spatial_dropout1d_3 (Spatia (None, 500, 300) 0
lDropout1D)
lstm_1 (LSTM) (None, 500, 25) 32600
batch_normalization_3 (Batc (None, 500, 25) 100
hNormalization)
dropout_3 (Dropout) (None, 500, 25) 0
global_max_pooling1d_3 (Glo (None, 25) 0
balMaxPooling1D)
dense_6 (Dense) (None, 50) 1300
dense_7 (Dense) (None, 15) 765
=================================================================
Total params: 30,034,765
Trainable params: 34,715
Non-trainable params: 30,000,050
_________________________________________________________________
LSTM_FastText_model_fit = LSTM_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15
2023-04-02 14:55:18.095303: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:18.097359: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:18.099749: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 14:55:19.058546: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:55:19.061164: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:55:19.063136: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
504/504 [==============================] - ETA: 0s - loss: 0.1861 - accuracy: 0.5581 - mean_pred: 0.0816 - fmeasure: 0.4369 - precision: 0.6876 - recall: 0.3245
2023-04-02 14:59:48.398786: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 14:59:48.400815: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 14:59:48.402662: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
504/504 [==============================] - 289s 568ms/step - loss: 0.1861 - accuracy: 0.5581 - mean_pred: 0.0816 - fmeasure: 0.4369 - precision: 0.6876 - recall: 0.3245 - val_loss: 0.2228 - val_accuracy: 0.6085 - val_mean_pred: 0.1569 - val_fmeasure: 0.5065 - val_precision: 0.7367 - val_recall: 0.3863 Epoch 2/15 504/504 [==============================] - 284s 563ms/step - loss: 0.1730 - accuracy: 0.5985 - mean_pred: 0.0810 - fmeasure: 0.4982 - precision: 0.7190 - recall: 0.3824 - val_loss: 0.2046 - val_accuracy: 0.6145 - val_mean_pred: 0.1415 - val_fmeasure: 0.5451 - val_precision: 0.6981 - val_recall: 0.4475 Epoch 3/15 504/504 [==============================] - 291s 577ms/step - loss: 0.1702 - accuracy: 0.6073 - mean_pred: 0.0810 - fmeasure: 0.5131 - precision: 0.7199 - recall: 0.3998 - val_loss: 0.2042 - val_accuracy: 0.6208 - val_mean_pred: 0.1398 - val_fmeasure: 0.5218 - val_precision: 0.7508 - val_recall: 0.4002 Epoch 4/15 504/504 [==============================] - 284s 564ms/step - loss: 0.1684 - accuracy: 0.6135 - mean_pred: 0.0810 - fmeasure: 0.5202 - precision: 0.7223 - recall: 0.4077 - val_loss: 0.1929 - val_accuracy: 0.6040 - val_mean_pred: 0.1236 - val_fmeasure: 0.5145 - val_precision: 0.7293 - val_recall: 0.3977 Epoch 5/15 504/504 [==============================] - 282s 560ms/step - loss: 0.1666 - accuracy: 0.6156 - mean_pred: 0.0810 - fmeasure: 0.5277 - precision: 0.7248 - recall: 0.4160 - val_loss: 0.1777 - val_accuracy: 0.6250 - val_mean_pred: 0.1056 - val_fmeasure: 0.4987 - val_precision: 0.7701 - val_recall: 0.3691 Epoch 6/15 504/504 [==============================] - 283s 561ms/step - loss: 0.1653 - accuracy: 0.6178 - mean_pred: 0.0810 - fmeasure: 0.5349 - precision: 0.7249 - recall: 0.4250 - val_loss: 0.1795 - val_accuracy: 0.6166 - val_mean_pred: 0.1066 - val_fmeasure: 0.5494 - val_precision: 0.7195 - val_recall: 0.4447 Epoch 7/15 504/504 [==============================] - 283s 561ms/step - loss: 0.1645 - accuracy: 0.6208 - mean_pred: 0.0809 - fmeasure: 0.5404 - precision: 0.7258 - recall: 0.4314 - val_loss: 0.1757 - val_accuracy: 0.6267 - val_mean_pred: 0.0984 - val_fmeasure: 0.4992 - val_precision: 0.7919 - val_recall: 0.3648 Epoch 8/15 504/504 [==============================] - 282s 560ms/step - loss: 0.1639 - accuracy: 0.6244 - mean_pred: 0.0809 - fmeasure: 0.5434 - precision: 0.7278 - recall: 0.4343 - val_loss: 0.1780 - val_accuracy: 0.6200 - val_mean_pred: 0.0997 - val_fmeasure: 0.5455 - val_precision: 0.7223 - val_recall: 0.4386 Epoch 9/15 504/504 [==============================] - 282s 559ms/step - loss: 0.1633 - accuracy: 0.6255 - mean_pred: 0.0809 - fmeasure: 0.5449 - precision: 0.7280 - recall: 0.4362 - val_loss: 0.1732 - val_accuracy: 0.6252 - val_mean_pred: 0.0950 - val_fmeasure: 0.5545 - val_precision: 0.7246 - val_recall: 0.4495 Epoch 10/15 504/504 [==============================] - 282s 559ms/step - loss: 0.1629 - accuracy: 0.6251 - mean_pred: 0.0809 - fmeasure: 0.5473 - precision: 0.7278 - recall: 0.4394 - val_loss: 0.1734 - val_accuracy: 0.6236 - val_mean_pred: 0.0881 - val_fmeasure: 0.5030 - val_precision: 0.7921 - val_recall: 0.3691 Epoch 11/15 504/504 [==============================] - 291s 578ms/step - loss: 0.1625 - accuracy: 0.6257 - mean_pred: 0.0809 - fmeasure: 0.5492 - precision: 0.7252 - recall: 0.4428 - val_loss: 0.1722 - val_accuracy: 0.6270 - val_mean_pred: 0.0908 - val_fmeasure: 0.5294 - val_precision: 0.7701 - val_recall: 0.4038 Epoch 12/15 504/504 [==============================] - 304s 602ms/step - loss: 0.1622 - accuracy: 0.6272 - mean_pred: 0.0808 - fmeasure: 0.5520 - precision: 0.7281 - recall: 0.4452 - val_loss: 0.1702 - val_accuracy: 0.6314 - val_mean_pred: 0.0860 - val_fmeasure: 0.5496 - val_precision: 0.7353 - val_recall: 0.4392 Epoch 13/15 504/504 [==============================] - 305s 606ms/step - loss: 0.1619 - accuracy: 0.6273 - mean_pred: 0.0809 - fmeasure: 0.5547 - precision: 0.7286 - recall: 0.4486 - val_loss: 0.1711 - val_accuracy: 0.6282 - val_mean_pred: 0.0862 - val_fmeasure: 0.5458 - val_precision: 0.7523 - val_recall: 0.4287 Epoch 14/15 504/504 [==============================] - 305s 605ms/step - loss: 0.1618 - accuracy: 0.6286 - mean_pred: 0.0809 - fmeasure: 0.5551 - precision: 0.7285 - recall: 0.4489 - val_loss: 0.1706 - val_accuracy: 0.6227 - val_mean_pred: 0.0864 - val_fmeasure: 0.5260 - val_precision: 0.7587 - val_recall: 0.4029 Epoch 15/15 504/504 [==============================] - 384s 762ms/step - loss: 0.1618 - accuracy: 0.6290 - mean_pred: 0.0809 - fmeasure: 0.5538 - precision: 0.7274 - recall: 0.4478 - val_loss: 0.1700 - val_accuracy: 0.6206 - val_mean_pred: 0.0836 - val_fmeasure: 0.5495 - val_precision: 0.7384 - val_recall: 0.4379
### Plot Training & Validation Accuracy with the Loss values of the LSTM-fastText Model# Plot training & validation accuracy values
plt.plot(LSTM_FastText_model_fit.history['accuracy'])
plt.plot(LSTM_FastText_model_fit.history['val_accuracy'])
plt.title('LSTM-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(LSTM_FastText_model_fit.history['loss'])
plt.plot(LSTM_FastText_model_fit.history['val_loss'])
plt.title('LSTM-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
#4 Bidirecitional LSTM with fastText
Bil_LSTM_FastText_model = Sequential([
Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False),
SpatialDropout1D(0.5),
#Bidirectional layer will enable our model to predict a missing word in a sequence,
#So, using this feature will enable the model to look at the context on both the left and the right.
Bidirectional(LSTM(25, return_sequences=True)),
#**batch normalization layer** normalizes the activations of the previous layer at each batch,
#i.e. applies a transformation that maintains the mean activation close to 0 and the activation standard deviation close to 1.
BatchNormalization(),
Dropout(0.5),
GlobalMaxPool1D(),
Dense(50, activation = 'relu'),
Dense(15, activation = 'sigmoid')
])
Bil_LSTM_FastText_model.compile(loss='binary_crossentropy', optimizer=Adam(0.01), metrics=['accuracy', mean_pred, fmeasure, precision, recall])
2023-04-02 16:09:09.108807: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:09.110715: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:09.112385: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:09.286877: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:09.350579: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:09.352680: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:09.354569: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
Bil_LSTM_FastText_model.summary()
Model: "sequential_4"
_________________________________________________________________
Layer (type) Output Shape Param #
=================================================================
embedding_4 (Embedding) (None, 500, 300) 30000000
spatial_dropout1d_4 (Spatia (None, 500, 300) 0
lDropout1D)
bidirectional (Bidirectiona (None, 500, 50) 65200
l)
batch_normalization_4 (Batc (None, 500, 50) 200
hNormalization)
dropout_4 (Dropout) (None, 500, 50) 0
global_max_pooling1d_4 (Glo (None, 50) 0
balMaxPooling1D)
dense_8 (Dense) (None, 50) 2550
dense_9 (Dense) (None, 15) 765
=================================================================
Total params: 30,068,715
Trainable params: 68,615
Non-trainable params: 30,000,100
_________________________________________________________________
Bil_LSTM_FastText_model_fit = Bil_LSTM_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15
2023-04-02 16:09:17.604221: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:17.606537: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:17.608798: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:17.781428: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:17.849680: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:17.851761: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:17.853625: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:18.572727: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:19.323679: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:19.326493: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:19.329047: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:19.497399: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:09:19.562745: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:09:19.565464: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:09:19.567264: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:09:20.235908: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
504/504 [==============================] - ETA: 0s - loss: 0.1875 - accuracy: 0.5636 - mean_pred: 0.0822 - fmeasure: 0.4517 - precision: 0.6835 - recall: 0.3415
2023-04-02 16:17:46.164026: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:17:46.165725: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:17:46.167560: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
2023-04-02 16:17:46.335129: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis' with dtype int32 and shape [1]
[[{{node gradients/ReverseV2_grad/ReverseV2/ReverseV2/axis}}]]
2023-04-02 16:17:46.392583: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_2_grad/concat/split_2/split_dim' with dtype int32
[[{{node gradients/split_2_grad/concat/split_2/split_dim}}]]
2023-04-02 16:17:46.394523: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_grad/concat/split/split_dim' with dtype int32
[[{{node gradients/split_grad/concat/split/split_dim}}]]
2023-04-02 16:17:46.396468: I tensorflow/core/common_runtime/executor.cc:1197] [/device:CPU:0] (DEBUG INFO) Executor start aborting (this does not indicate an error and you can ignore this message): INVALID_ARGUMENT: You must feed a value for placeholder tensor 'gradients/split_1_grad/concat/split_1/split_dim' with dtype int32
[[{{node gradients/split_1_grad/concat/split_1/split_dim}}]]
504/504 [==============================] - 547s 1s/step - loss: 0.1875 - accuracy: 0.5636 - mean_pred: 0.0822 - fmeasure: 0.4517 - precision: 0.6835 - recall: 0.3415 - val_loss: 0.2214 - val_accuracy: 0.6109 - val_mean_pred: 0.1587 - val_fmeasure: 0.4878 - val_precision: 0.7495 - val_recall: 0.3619 Epoch 2/15 504/504 [==============================] - 527s 1s/step - loss: 0.1710 - accuracy: 0.6042 - mean_pred: 0.0811 - fmeasure: 0.5109 - precision: 0.7179 - recall: 0.3983 - val_loss: 0.2104 - val_accuracy: 0.6173 - val_mean_pred: 0.1499 - val_fmeasure: 0.5163 - val_precision: 0.7568 - val_recall: 0.3923 Epoch 3/15 504/504 [==============================] - 512s 1s/step - loss: 0.1670 - accuracy: 0.6174 - mean_pred: 0.0811 - fmeasure: 0.5320 - precision: 0.7207 - recall: 0.4230 - val_loss: 0.1967 - val_accuracy: 0.6232 - val_mean_pred: 0.1339 - val_fmeasure: 0.5334 - val_precision: 0.7448 - val_recall: 0.4159 Epoch 4/15 504/504 [==============================] - 521s 1s/step - loss: 0.1647 - accuracy: 0.6221 - mean_pred: 0.0810 - fmeasure: 0.5424 - precision: 0.7229 - recall: 0.4353 - val_loss: 0.1826 - val_accuracy: 0.6298 - val_mean_pred: 0.1190 - val_fmeasure: 0.5625 - val_precision: 0.7251 - val_recall: 0.4598 Epoch 5/15 504/504 [==============================] - 441s 876ms/step - loss: 0.1630 - accuracy: 0.6268 - mean_pred: 0.0810 - fmeasure: 0.5486 - precision: 0.7273 - recall: 0.4413 - val_loss: 0.1834 - val_accuracy: 0.6209 - val_mean_pred: 0.1188 - val_fmeasure: 0.5645 - val_precision: 0.7198 - val_recall: 0.4648 Epoch 6/15 504/504 [==============================] - 591s 1s/step - loss: 0.1616 - accuracy: 0.6310 - mean_pred: 0.0810 - fmeasure: 0.5561 - precision: 0.7275 - recall: 0.4510 - val_loss: 0.1775 - val_accuracy: 0.6342 - val_mean_pred: 0.1107 - val_fmeasure: 0.5636 - val_precision: 0.7309 - val_recall: 0.4590 Epoch 7/15 504/504 [==============================] - 566s 1s/step - loss: 0.1613 - accuracy: 0.6326 - mean_pred: 0.0811 - fmeasure: 0.5599 - precision: 0.7292 - recall: 0.4551 - val_loss: 0.1731 - val_accuracy: 0.6293 - val_mean_pred: 0.0998 - val_fmeasure: 0.5690 - val_precision: 0.7218 - val_recall: 0.4699 Epoch 8/15 504/504 [==============================] - 544s 1s/step - loss: 0.1602 - accuracy: 0.6352 - mean_pred: 0.0809 - fmeasure: 0.5627 - precision: 0.7347 - recall: 0.4568 - val_loss: 0.1716 - val_accuracy: 0.6371 - val_mean_pred: 0.0998 - val_fmeasure: 0.5758 - val_precision: 0.7292 - val_recall: 0.4761 Epoch 9/15 504/504 [==============================] - 491s 974ms/step - loss: 0.1595 - accuracy: 0.6375 - mean_pred: 0.0809 - fmeasure: 0.5649 - precision: 0.7331 - recall: 0.4603 - val_loss: 0.1705 - val_accuracy: 0.6377 - val_mean_pred: 0.0950 - val_fmeasure: 0.5689 - val_precision: 0.7384 - val_recall: 0.4631 Epoch 10/15 504/504 [==============================] - 522s 1s/step - loss: 0.1590 - accuracy: 0.6395 - mean_pred: 0.0809 - fmeasure: 0.5658 - precision: 0.7375 - recall: 0.4597 - val_loss: 0.1671 - val_accuracy: 0.6398 - val_mean_pred: 0.0892 - val_fmeasure: 0.5690 - val_precision: 0.7355 - val_recall: 0.4644 Epoch 11/15 504/504 [==============================] - 521s 1s/step - loss: 0.1585 - accuracy: 0.6407 - mean_pred: 0.0809 - fmeasure: 0.5692 - precision: 0.7371 - recall: 0.4643 - val_loss: 0.1699 - val_accuracy: 0.6432 - val_mean_pred: 0.0908 - val_fmeasure: 0.5788 - val_precision: 0.7322 - val_recall: 0.4789 Epoch 12/15 504/504 [==============================] - 496s 984ms/step - loss: 0.1578 - accuracy: 0.6434 - mean_pred: 0.0809 - fmeasure: 0.5725 - precision: 0.7379 - recall: 0.4683 - val_loss: 0.1698 - val_accuracy: 0.6409 - val_mean_pred: 0.0834 - val_fmeasure: 0.5735 - val_precision: 0.7382 - val_recall: 0.4692 Epoch 13/15 504/504 [==============================] - 481s 956ms/step - loss: 0.1578 - accuracy: 0.6435 - mean_pred: 0.0808 - fmeasure: 0.5731 - precision: 0.7371 - recall: 0.4694 - val_loss: 0.1718 - val_accuracy: 0.6385 - val_mean_pred: 0.0818 - val_fmeasure: 0.5664 - val_precision: 0.7423 - val_recall: 0.4583 Epoch 14/15 504/504 [==============================] - 476s 944ms/step - loss: 0.1577 - accuracy: 0.6457 - mean_pred: 0.0808 - fmeasure: 0.5752 - precision: 0.7392 - recall: 0.4712 - val_loss: 0.1734 - val_accuracy: 0.6454 - val_mean_pred: 0.0812 - val_fmeasure: 0.5868 - val_precision: 0.7276 - val_recall: 0.4921
# Plot training & validation accuracy values
plt.plot(Bil_LSTM_FastText_model_fit.history['accuracy'])
plt.plot(Bil_LSTM_FastText_model_fit.history['val_accuracy'])
plt.title('Bidirecitonal LSTM-fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(Bil_LSTM_FastText_model_fit.history['loss'])
plt.plot(Bil_LSTM_FastText_model_fit.history['val_loss'])
plt.title('Bidirecitonal LSTM-fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
#5 Gated Recurrent (GRU) with fastText
sequence_input = Input(shape=(max_len, ))
model = Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False)(sequence_input)
model = SpatialDropout1D(0.2)(model)
model = GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1)(model)
model = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform")(model)
avg_pool = GlobalAveragePooling1D()(model)
max_pool = GlobalMaxPooling1D()(model)
model = concatenate([avg_pool, max_pool])
preds = Dense(15, activation="sigmoid")(model)
GRU_FastText_model = Model(sequence_input, preds)
GRU_FastText_model.compile(loss='binary_crossentropy',optimizer="Adam",metrics=['accuracy', mean_pred, fmeasure, precision, recall])
GRU_FastText_model.summary()
Model: "model" __________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== Total params: 30,191,695 Trainable params: 191,695 Non-trainable params: 30,000,000 __________________________________________________________________________________________________
GRU_FastText_model_fit = GRU_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15 504/504 [==============================] - 1208s 2s/step - loss: 0.1855 - accuracy: 0.5680 - mean_pred: 0.0867 - fmeasure: 0.4439 - precision: 0.7107 - recall: 0.3294 - val_loss: 0.1684 - val_accuracy: 0.6199 - val_mean_pred: 0.0831 - val_fmeasure: 0.5356 - val_precision: 0.7314 - val_recall: 0.4229 Epoch 2/15 504/504 [==============================] - 1215s 2s/step - loss: 0.1628 - accuracy: 0.6320 - mean_pred: 0.0809 - fmeasure: 0.5434 - precision: 0.7489 - recall: 0.4276 - val_loss: 0.1614 - val_accuracy: 0.6424 - val_mean_pred: 0.0829 - val_fmeasure: 0.5405 - val_precision: 0.7725 - val_recall: 0.4162 Epoch 3/15 504/504 [==============================] - 1024s 2s/step - loss: 0.1560 - accuracy: 0.6531 - mean_pred: 0.0809 - fmeasure: 0.5707 - precision: 0.7584 - recall: 0.4584 - val_loss: 0.1566 - val_accuracy: 0.6583 - val_mean_pred: 0.0827 - val_fmeasure: 0.5755 - val_precision: 0.7566 - val_recall: 0.4648 Epoch 4/15 504/504 [==============================] - 1089s 2s/step - loss: 0.1512 - accuracy: 0.6671 - mean_pred: 0.0808 - fmeasure: 0.5899 - precision: 0.7670 - recall: 0.4801 - val_loss: 0.1538 - val_accuracy: 0.6662 - val_mean_pred: 0.0808 - val_fmeasure: 0.5903 - val_precision: 0.7543 - val_recall: 0.4853 Epoch 5/15 504/504 [==============================] - 979s 2s/step - loss: 0.1472 - accuracy: 0.6796 - mean_pred: 0.0808 - fmeasure: 0.6053 - precision: 0.7736 - recall: 0.4980 - val_loss: 0.1528 - val_accuracy: 0.6707 - val_mean_pred: 0.0788 - val_fmeasure: 0.5767 - val_precision: 0.7782 - val_recall: 0.4585 Epoch 6/15 504/504 [==============================] - 964s 2s/step - loss: 0.1434 - accuracy: 0.6898 - mean_pred: 0.0808 - fmeasure: 0.6197 - precision: 0.7834 - recall: 0.5136 - val_loss: 0.1513 - val_accuracy: 0.6732 - val_mean_pred: 0.0840 - val_fmeasure: 0.6106 - val_precision: 0.7490 - val_recall: 0.5157 Epoch 7/15 504/504 [==============================] - 963s 2s/step - loss: 0.1400 - accuracy: 0.6994 - mean_pred: 0.0808 - fmeasure: 0.6324 - precision: 0.7902 - recall: 0.5278 - val_loss: 0.1518 - val_accuracy: 0.6671 - val_mean_pred: 0.0790 - val_fmeasure: 0.5967 - val_precision: 0.7670 - val_recall: 0.4887 Epoch 8/15 504/504 [==============================] - 1031s 2s/step - loss: 0.1365 - accuracy: 0.7069 - mean_pred: 0.0808 - fmeasure: 0.6441 - precision: 0.7979 - recall: 0.5407 - val_loss: 0.1511 - val_accuracy: 0.6742 - val_mean_pred: 0.0836 - val_fmeasure: 0.6140 - val_precision: 0.7482 - val_recall: 0.5209 Epoch 9/15 504/504 [==============================] - 1104s 2s/step - loss: 0.1331 - accuracy: 0.7137 - mean_pred: 0.0808 - fmeasure: 0.6544 - precision: 0.8044 - recall: 0.5523 - val_loss: 0.1508 - val_accuracy: 0.6786 - val_mean_pred: 0.0824 - val_fmeasure: 0.6154 - val_precision: 0.7537 - val_recall: 0.5203 Epoch 10/15 504/504 [==============================] - 998s 2s/step - loss: 0.1298 - accuracy: 0.7236 - mean_pred: 0.0808 - fmeasure: 0.6659 - precision: 0.8125 - recall: 0.5647 - val_loss: 0.1537 - val_accuracy: 0.6734 - val_mean_pred: 0.0774 - val_fmeasure: 0.6114 - val_precision: 0.7580 - val_recall: 0.5127 Epoch 11/15 504/504 [==============================] - 975s 2s/step - loss: 0.1263 - accuracy: 0.7307 - mean_pred: 0.0807 - fmeasure: 0.6753 - precision: 0.8182 - recall: 0.5755 - val_loss: 0.1531 - val_accuracy: 0.6716 - val_mean_pred: 0.0849 - val_fmeasure: 0.6230 - val_precision: 0.7308 - val_recall: 0.5432 Epoch 12/15 504/504 [==============================] - 969s 2s/step - loss: 0.1230 - accuracy: 0.7378 - mean_pred: 0.0808 - fmeasure: 0.6860 - precision: 0.8246 - recall: 0.5878 - val_loss: 0.1546 - val_accuracy: 0.6706 - val_mean_pred: 0.0783 - val_fmeasure: 0.6178 - val_precision: 0.7480 - val_recall: 0.5267 Epoch 13/15 504/504 [==============================] - 973s 2s/step - loss: 0.1195 - accuracy: 0.7453 - mean_pred: 0.0807 - fmeasure: 0.6959 - precision: 0.8316 - recall: 0.5988 - val_loss: 0.1564 - val_accuracy: 0.6681 - val_mean_pred: 0.0807 - val_fmeasure: 0.6189 - val_precision: 0.7389 - val_recall: 0.5328
# Plot training & validation accuracy values
plt.plot(GRU_FastText_model_fit.history['accuracy'])
plt.plot(GRU_FastText_model_fit.history['val_accuracy'])
plt.title('Gated Recurrent Unit (GRU) with fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(GRU_FastText_model_fit.history['loss'])
plt.plot(GRU_FastText_model_fit.history['val_loss'])
plt.title('Bidirectional Gated Recurrent Unit (GRU) with fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
#6 Bidirectional Gated Recurrent (GRU) with fastText
sequence_input = Input(shape=(max_len, ))
model = Embedding(input_dim =fastText_embedding_matrix.shape[0], input_length=max_len, output_dim=fastText_embedding_matrix.shape[1],weights=[fastText_embedding_matrix], trainable=False)(sequence_input)
model = SpatialDropout1D(0.2)(model)
model = Bidirectional(GRU(128, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(model)
model = Conv1D(64, kernel_size = 3, padding = "valid", kernel_initializer = "glorot_uniform")(model)
avg_pool = GlobalAveragePooling1D()(model)
max_pool = GlobalMaxPooling1D()(model)
model = concatenate([avg_pool, max_pool])
preds = Dense(15, activation="sigmoid")(model)
Bil_GRU_FastText_model = Model(sequence_input, preds)
Bil_GRU_FastText_model.compile(loss='binary_crossentropy',optimizer="Adam",metrics=['accuracy', mean_pred, fmeasure, precision, recall])
Bil_GRU_FastText_model.summary()
Model: "model_1" __________________________________________________________________________________________________ Layer (type) Output Shape Param # Connected to ================================================================================================== Total params: 30,381,391 Trainable params: 381,391 Non-trainable params: 30,000,000 __________________________________________________________________________________________________
Bil_GRU_FastText_model_fit = Bil_GRU_FastText_model.fit(X_tra, y_tra, batch_size=batch_size2, epochs=num_epochs, validation_data=(X_val, y_val), callbacks=[early])
Epoch 1/15 504/504 [==============================] - 1815s 4s/step - loss: 0.1802 - accuracy: 0.5886 - mean_pred: 0.0856 - fmeasure: 0.4758 - precision: 0.7180 - recall: 0.3633 - val_loss: 0.1641 - val_accuracy: 0.6319 - val_mean_pred: 0.0809 - val_fmeasure: 0.5255 - val_precision: 0.7631 - val_recall: 0.4011 Epoch 2/15 504/504 [==============================] - 1877s 4s/step - loss: 0.1591 - accuracy: 0.6429 - mean_pred: 0.0810 - fmeasure: 0.5585 - precision: 0.7554 - recall: 0.4441 - val_loss: 0.1575 - val_accuracy: 0.6562 - val_mean_pred: 0.0821 - val_fmeasure: 0.5509 - val_precision: 0.7838 - val_recall: 0.4252 Epoch 3/15 504/504 [==============================] - 1963s 4s/step - loss: 0.1521 - accuracy: 0.6650 - mean_pred: 0.0809 - fmeasure: 0.5867 - precision: 0.7668 - recall: 0.4759 - val_loss: 0.1535 - val_accuracy: 0.6653 - val_mean_pred: 0.0785 - val_fmeasure: 0.5911 - val_precision: 0.7572 - val_recall: 0.4851 Epoch 4/15 504/504 [==============================] - 2284s 5s/step - loss: 0.1466 - accuracy: 0.6806 - mean_pred: 0.0808 - fmeasure: 0.6080 - precision: 0.7769 - recall: 0.5003 - val_loss: 0.1514 - val_accuracy: 0.6743 - val_mean_pred: 0.0809 - val_fmeasure: 0.6068 - val_precision: 0.7547 - val_recall: 0.5077 Epoch 5/15 504/504 [==============================] - 2272s 5s/step - loss: 0.1418 - accuracy: 0.6950 - mean_pred: 0.0808 - fmeasure: 0.6264 - precision: 0.7879 - recall: 0.5206 - val_loss: 0.1502 - val_accuracy: 0.6765 - val_mean_pred: 0.0766 - val_fmeasure: 0.6078 - val_precision: 0.7608 - val_recall: 0.5066 Epoch 6/15 504/504 [==============================] - 2022s 4s/step - loss: 0.1367 - accuracy: 0.7065 - mean_pred: 0.0808 - fmeasure: 0.6433 - precision: 0.7978 - recall: 0.5397 - val_loss: 0.1501 - val_accuracy: 0.6742 - val_mean_pred: 0.0841 - val_fmeasure: 0.6140 - val_precision: 0.7509 - val_recall: 0.5197 Epoch 7/15 504/504 [==============================] - 2006s 4s/step - loss: 0.1315 - accuracy: 0.7189 - mean_pred: 0.0808 - fmeasure: 0.6602 - precision: 0.8077 - recall: 0.5588 - val_loss: 0.1497 - val_accuracy: 0.6771 - val_mean_pred: 0.0834 - val_fmeasure: 0.6204 - val_precision: 0.7449 - val_recall: 0.5319 Epoch 8/15 504/504 [==============================] - 1980s 4s/step - loss: 0.1261 - accuracy: 0.7318 - mean_pred: 0.0808 - fmeasure: 0.6753 - precision: 0.8172 - recall: 0.5760 - val_loss: 0.1531 - val_accuracy: 0.6745 - val_mean_pred: 0.0811 - val_fmeasure: 0.6133 - val_precision: 0.7510 - val_recall: 0.5186 Epoch 9/15 504/504 [==============================] - 1987s 4s/step - loss: 0.1205 - accuracy: 0.7433 - mean_pred: 0.0808 - fmeasure: 0.6919 - precision: 0.8284 - recall: 0.5946 - val_loss: 0.1525 - val_accuracy: 0.6710 - val_mean_pred: 0.0788 - val_fmeasure: 0.6110 - val_precision: 0.7510 - val_recall: 0.5155 Epoch 10/15 504/504 [==============================] - 1948s 4s/step - loss: 0.1149 - accuracy: 0.7569 - mean_pred: 0.0808 - fmeasure: 0.7084 - precision: 0.8379 - recall: 0.6142 - val_loss: 0.1560 - val_accuracy: 0.6729 - val_mean_pred: 0.0802 - val_fmeasure: 0.6188 - val_precision: 0.7432 - val_recall: 0.5305 Epoch 11/15 504/504 [==============================] - 1880s 4s/step - loss: 0.1088 - accuracy: 0.7699 - mean_pred: 0.0807 - fmeasure: 0.7266 - precision: 0.8489 - recall: 0.6358 - val_loss: 0.1604 - val_accuracy: 0.6590 - val_mean_pred: 0.0846 - val_fmeasure: 0.6157 - val_precision: 0.7158 - val_recall: 0.5407
# Plot training & validation accuracy values
plt.plot(Bil_GRU_FastText_model_fit.history['accuracy'])
plt.plot(Bil_GRU_FastText_model_fit.history['val_accuracy'])
plt.title(' Bidirectional Gated Recurrent Unit (GRU) with fastText Model accuracy')
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Training Accuracy', 'Validation Accuracy'], loc='upper left')
plt.show()
# Plot training & validation loss values
plt.plot(Bil_GRU_FastText_model_fit.history['loss'])
plt.plot(Bil_GRU_FastText_model_fit.history['val_loss'])
plt.title('Bidirectional Gated Recurrent Unit (GRU) with fastText Model loss')
plt.ylabel('Loss')
plt.xlabel('Epoch')
plt.legend(['Training Loss', 'Validation Loss'], loc='lower right')
plt.show()
from chart_studio import plotly
import plotly.offline as py
import plotly.graph_objs as go
#Computing the highest of the evaluation matrics (per model)
trace = go.Table(
header=dict(values=['Model', 'Loss', 'Accuracy', 'mean_pred', 'F-Measure', 'Precision', 'Recall'],
line = dict(color='#7D7F80'),
fill = dict(color='#a1c3d1'),
align = ['left'] * 5),
cells=dict(values=[['CNN-fastText', 'RNNs-fastText', 'LSTM-fastText', ' BiLSTM-fastText', 'GRU-fastText', 'BiGRU-fastText'],
[
#Loss Evaluation
round(np.max(CNN_FastText_model_fit.history['loss']), 3), round(np.max(RNN_FastText_model_fit.history['loss']), 3),
round(np.max(LSTM_FastText_model_fit.history['loss']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['loss']), 3), round(np.max(GRU_FastText_model_fit.history['loss']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['loss']), 3)],
#Accuracy Evaluation
[round(np.max(CNN_FastText_model_fit.history['accuracy']), 3), round(np.max(RNN_FastText_model_fit.history['accuracy']), 3),
round(np.max(LSTM_FastText_model_fit.history['accuracy']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['accuracy']), 3), round(np.max(GRU_FastText_model_fit.history['accuracy']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['accuracy']), 3)],
#mean_pred Evaluation
[round(np.max(CNN_FastText_model_fit.history['mean_pred']), 3), round(np.max(RNN_FastText_model_fit.history['mean_pred']), 3),
round(np.max(LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.max(GRU_FastText_model_fit.history['mean_pred']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['mean_pred']), 3)],
#F1-Measure Evaluation fmeasure
[round(np.max(CNN_FastText_model_fit.history['fmeasure']), 3), round(np.max(RNN_FastText_model_fit.history['fmeasure']), 3),
round(np.max(LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.max(GRU_FastText_model_fit.history['fmeasure']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['fmeasure']), 3)],
#Precision Evaluation precision
[round(np.max(CNN_FastText_model_fit.history['precision']), 3), round(np.max(RNN_FastText_model_fit.history['precision']), 3),
round(np.max(LSTM_FastText_model_fit.history['precision']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['precision']), 3), round(np.max(GRU_FastText_model_fit.history['precision']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['precision']), 3)],
#Recall Evaluation
[round(np.max(CNN_FastText_model_fit.history['recall']), 3), round(np.max(RNN_FastText_model_fit.history['recall']), 3),
round(np.max(LSTM_FastText_model_fit.history['recall']), 3), round(np.max(Bil_LSTM_FastText_model_fit.history['recall']), 3), round(np.max(GRU_FastText_model_fit.history['recall']), 3),
round(np.max(Bil_GRU_FastText_model_fit.history['recall']), 3)]
],
line = dict(color='#7D7F80'),
fill = dict(color='#EDFAFF'),
align = ['left'] * 5))
layout = dict(width=800, height=400)
data = [trace]
fig = dict(data=data, layout=layout)
py.iplot(data, filename = 'FastTexttrained_embedding_with the max of the evaluation matrics (per model) _table')
#Computing the mean of the evaluation matrics (per model)
trace = go.Table(
header=dict(values=['Model', 'Loss', 'Accuracy', 'mean_pred', 'F-Measure', 'Precision', 'Recall'],
line = dict(color='#7D7F80'),
fill = dict(color='#a1c3d1'),
align = ['left'] * 5),
cells=dict(values=[['CNN-fastText', 'RNNs-fastText', 'LSTM-fastText', ' BiLSTM-fastText', 'GRU-fastText', 'BiGRU-fastText'],
[
#Loss Evaluation
round(np.mean(CNN_FastText_model_fit.history['loss']), 3), round(np.mean(RNN_FastText_model_fit.history['loss']), 3),
round(np.mean(LSTM_FastText_model_fit.history['loss']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['loss']), 3), round(np.mean(GRU_FastText_model_fit.history['loss']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['loss']), 3)],
#Accuracy Evaluation
[round(np.mean(CNN_FastText_model_fit.history['accuracy']), 3), round(np.mean(RNN_FastText_model_fit.history['accuracy']), 3),
round(np.mean(LSTM_FastText_model_fit.history['accuracy']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['accuracy']), 3), round(np.mean(GRU_FastText_model_fit.history['accuracy']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['accuracy']), 3)],
#mean_pred Evaluation
[round(np.mean(CNN_FastText_model_fit.history['mean_pred']), 3), round(np.mean(RNN_FastText_model_fit.history['mean_pred']), 3),
round(np.mean(LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['mean_pred']), 3), round(np.mean(GRU_FastText_model_fit.history['mean_pred']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['mean_pred']), 3)],
#F1-Measure Evaluation fmeasure
[round(np.mean(CNN_FastText_model_fit.history['fmeasure']), 3), round(np.mean(RNN_FastText_model_fit.history['fmeasure']), 3),
round(np.mean(LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['fmeasure']), 3), round(np.mean(GRU_FastText_model_fit.history['fmeasure']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['fmeasure']), 3)],
#Precision Evaluation precision
[round(np.mean(CNN_FastText_model_fit.history['precision']), 3), round(np.mean(RNN_FastText_model_fit.history['precision']), 3),
round(np.mean(LSTM_FastText_model_fit.history['precision']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['precision']), 3), round(np.mean(GRU_FastText_model_fit.history['precision']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['precision']), 3)],
#Recall Evaluation
[round(np.mean(CNN_FastText_model_fit.history['recall']), 3), round(np.mean(RNN_FastText_model_fit.history['recall']), 3),
round(np.mean(LSTM_FastText_model_fit.history['recall']), 3), round(np.mean(Bil_LSTM_FastText_model_fit.history['recall']), 3), round(np.mean(GRU_FastText_model_fit.history['recall']), 3),
round(np.mean(Bil_GRU_FastText_model_fit.history['recall']), 3)]
],
line = dict(color='#7D7F80'),
fill = dict(color='#EDFAFF'),
align = ['left'] * 5))
layout = dict(width=800, height=400)
data = [trace]
fig = dict(data=data, layout=layout)
py.iplot(data, filename = 'FastTexttrained_embedding_with the Mean value of the evaluation matrics (per model) _table')
#Score Confusion Table
# concat all training, validation and testing accuracy scores
CNN_FastTextAccuracy = ['Convolutional Neural Network (CNN) with fastText',
np.max(CNN_FastTextmodel_fit.history['accuracy']),
np.max(CNN_FastTextmodel_fit.history['val_accuracy']),
CNN_FastTexttest_score[1]]
RNN_FastTextAccuracy = ['Recurrent Neural Networks (RNNs) with fastText',
np.max(RNN_FastTextmodel_fit.history['accuracy']),
np.max(RNN_FastTextmodel_fit.history['val_accuracy']),
RNN_FastTexttest_score[1]]
LSTM_FastTextAccuracy = ['LSTM with fastText',
np.max(LSTM_FastTextmodel_fit.history['accuracy']),
np.max(LSTM_FastTextmodel_fit.history['val_accuracy']),
LSTM_FastTexttest_score[1]]
Bidirectional_LSTM_FastTextAccuracy = ['Bidirectional LSTM with fastText',
np.max(Bil_LSTM_FastTextmodel_fit.history['accuracy']),
np.max(Bil_LSTM_FastTextmodel_fit.history['val_accuracy']),
Bil_LSTM_FastTexttest_score[1]]
GRU_FastTextAccuracy = ['GRU with fastText',
np.max(GRU_FastTextmodel_fit.history['accuracy']),
np.max(GRU_FastTextmodel_fit.history['val_accuracy']),
GRU_FastTexttest_score[1]]
Bidirectional_GRU_FastTextAccuracy = ['Bidirectional GRU with fastText',
np.max(Bil_GRU_FastTextmodel_fit.history['accuracy']),
np.max(Bil_GRU_FastTextmodel_fit.history['val_accuracy']),
Bil_GRU_FastTexttest_score[1]]
# create dataframe
experimentalResult = pd.DataFrame([CNN_FastTextAccuracy])
# append all other scores
experimentalResult = experimentalResult.append([CNN_FastTextAccuracy, RNN_FastTextAccuracy, LSTM_FastTextAccuracy, Bidirectional_LSTM_FastTextAccuracy,
GRU_FastTextAccuracy, Bidirectional_GRU_FastTextAccuracy])
# beautify the new dataframe
experimentalResult.columns = ['Model', 'Training Accuracy', 'Validation Accuracy', 'Testing Accuracy']
experimentalResult.set_index(['Model'], inplace=True)
experimentalResult
print(experimentalResult)